#! /bin/sh
# PCP QA Test No. 282
# Error handling in pmcd & pmlogger rc scripts
#
# Note: the core of this test needs to use the explicit init scripts,
#	not the _service wrapper so there is no chance of systemd
#	getting in the way
#
# Copyright (c) 2014 Red Hat.
# Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
#

seq=`basename $0`
echo "QA output created by $seq"

# get standard environment, filters and checks
. ./common.product
. ./common.filter
. ./common.check

case $PCP_PLATFORM
in
    linux|darwin|solaris|freebsd|openbsd)
	;;
    *)
	_notrun "$PCP_PLATFORM is not yet supported in test $seq"
	;;
esac

status=1	# failure is the default!
_needclean=true
sleepy=false

LOCALHOST=`hostname`
LOGGING_DIR="$PCP_ARCHIVE_DIR"
CHECK=`which pmlogger_check`
[ -z "$CHECK" -o ! -x "$CHECK" ] && \
    _notrun "Cannot find an executable pmlogger_check: \"$CHECK\""

_poke_pmcd()
{
    # Note, need to use explict pmcd, then pmlogger to avoid this maybe
    # going off into systemd land via the pcp init script wrapper
    #
    ( $sudo $PCP_RC_DIR/pmcd restart; $sudo $PCP_RC_DIR/pmlogger restart ) 2>&1 \
    | tee -a $seq_full \
    | sed \
	-e '/ root .* .usr.etc.pmcd /s/.*/-- ps output line --/' \
	-e '/UID .* PID /d' \
	-e '/USER .* PID /d' \
	-e "s/$LOCALHOST/HOST/g" \
	-e 's|local:/*|HOST|g' \
    | _filter_pcp_start \
    | _filter_pmcd_log \
    | _filter_cron_scripts \
    | sed \
	-e 's/\(install the pcp.sw.base\) .*/\1 ... PCP 2.0 or later .../' \
	-e 's/.*\(consult PMLOGGER.CHECK(1), else\)/        ... \1/' \
	-e '/root.*.PCP_BINADM_DIR.pmcd/s/.*/-- ps output line --/' \
	-e '/pcp.*.PCP_BINADM_DIR.pmcd/s/.*/-- ps output line --/' \
	-e '/^failed (status=1)/d' \
	-e '/^Waiting for pmcd/s/\.\.\.[. ]*$/DOTS/' \
	-e '/^\.\.\.[. ]*$/d' \
	-e '/^$/d'

    can_wait=5
    i=0
    while [ $i -lt $can_wait ]
    do
	clients=`pmprobe pmcd.numclients 2>/dev/null | sed -e 's/.* //'`
	if [ $i -eq 0 ]
	then
	    echo >>$seq_full
	    echo "+ pmprobe pmcd.numclients" >>$seq_full
	    pmprobe pmcd.numclients >>$seq_full 2>&1
	    echo "clients=\"$clients\"" >>$seq_full
	fi
	[ ! -z "$clients" ] && [ "$clients" -gt 0 ] && break
	sleep 1
	i=`expr $i + 1`
    done

    $PCP_ECHO_PROG $PCP_ECHO_N "=== probe pmcd: ""$PCP_ECHO_C"
    x=`pmprobe pmcd.control.debug 2>&1 | sed -e "s/$LOCALHOST/HOST/g" -e 's|local:/*|HOST|g'`
    if [ "$x" = "pmcd.control.debug 1" ]
    then
	$PCP_ECHO_PROG OK
    else
	$PCP_ECHO_PROG "$x"
    fi
}

_poke_pmcd_n_logger()
{
    pmafm $LOGGING_DIR/$LOCALHOST/Latest remove >$tmp.cmd 2>&1 \
    && $sudo sh $tmp.cmd
    _poke_pmcd

    can_wait=5
    i=0
    while [ $i -lt $can_wait ]
    do
	if pmlc -P </dev/null 2>&1 | grep "Connected to.* pmlogger" >/dev/null
	then
	    sleep 2
	    break
	fi
	sleep 2
	i=`expr $i + 1`
    done

    $PCP_ECHO_PROG $PCP_ECHO_N "=== probe pmlogger: ""$PCP_ECHO_C"
    x=`echo connect primary | pmlc 2>&1 | sed -e "s/$LOCALHOST/HOST/g" -e 's|local:/*|HOST|g'`
    sleep 2
    if [ -z "$x" ]
    then
	$PCP_ECHO_PROG OK
    else
	$PCP_ECHO_PROG "$x"
        if $sleepy
        then
            echo go check the logs while I wait a couple of minutes
            sleep 240
        fi
    fi
}

_interrupt()
{
    status=1
}

# Save config files and pmlogger_check, but leave the originals in place
#
_save_setup()
{
    for file in $PCP_PMCDCONF_PATH $CHECK $PCP_PMLOGGEROPTIONS_PATH
    do
	if [ -f $file ]
	then
	    $sudo rm -f $file.$seq
	    $sudo mv $file $file.$seq
	    $sudo cp -p $file.$seq $file
	fi
    done
    _disable_loggers || _exit 1
}

_cleanup()
{
    trap "" 0 1 2 3 15
    if $_needclean
    then
	echo | tee -a $seq_full
	echo "Restore the state of the universe ..."
	for file in $PCP_PMCDCONF_PATH $CHECK $PCP_PMLOGGEROPTIONS_PATH
	do
	    if [ -f $file.$seq ]
	    then
		$sudo rm -f $file
		$sudo mv $file.$seq $file
	    fi
	done
	_restore_loggers
	if [ -f $PCP_PMCD_PROG.$seq ]
	then
	    $sudo rm -f $PCP_PMCD_PROG
	    $sudo mv $PCP_PMCD_PROG.$seq $PCP_PMCD_PROG
	fi

	pmafm $LOGGING_DIR/$LOCALHOST/Latest remove >$tmp.cmd 2>&1 \
	&& $sudo sh $tmp.cmd
	_service pmcd restart 2>&1 | _filter_pcp_restart
	_wait_for_pmcd
	_restore_auto_restart pmcd
	_service pmlogger restart 2>&1 | _filter_pcp_restart
	_wait_for_pmlogger
	_needclean=false
    fi
    $sudo rm -fr $tmp.*
    exit $status
}

trap "_cleanup" 0
trap "_interrupt; _cleanup" 1 2 3 15

touch $seq_full

_exercise_rc_scripts()
{
    echo | tee -a $seq_full
    echo '=== expect error: no $PCP_PMCDCONF_PATH ==='
    $sudo rm -f $PCP_PMCDCONF_PATH
    _poke_pmcd

    if [ $PCP_PLATFORM = linux ]
    then
	cat <<End-of-File >$tmp.out
# Installed by PCP QA test $seq on `date`
linux	60	dso	linux_init	$PCP_PMDAS_DIR/linux/pmda_linux.so
pmcd	2	dso	pmcd_init	$PCP_PMDAS_DIR/pmcd/pmda_pmcd.so
End-of-File
    elif [ $PCP_PLATFORM = darwin ]
    then
	cat <<End-of-File >$tmp.out
# Installed by PCP QA test $seq on `date`
darwin	78	dso	darwin_init	$PCP_PMDAS_DIR/darwin/pmda_darwin.dylib
pmcd	2	dso	pmcd_init	$PCP_PMDAS_DIR/pmcd/pmda_pmcd.dylib
End-of-File
    elif [ $PCP_PLATFORM = solaris ]
    then
	cat <<End-of-File >$tmp.out
# Installed by PCP QA test $seq on `date`
solaris	75	dso	solaris_init	$PCP_PMDAS_DIR/solaris/pmda_solaris.so
pmcd	2	dso	pmcd_init	$PCP_PMDAS_DIR/pmcd/pmda_pmcd.so
End-of-File
    elif [ $PCP_PLATFORM = freebsd ]
    then
	cat <<End-of-File >$tmp.out
# Installed by PCP QA test $seq on `date`
freebsd	85	dso	freebsd_init	$PCP_PMDAS_DIR/freebsd/pmda_freebsd.so
pmcd	2	dso	pmcd_init	$PCP_PMDAS_DIR/pmcd/pmda_pmcd.so
End-of-File
    elif [ $PCP_PLATFORM = openbsd ]
    then
	cat <<End-of-File >$tmp.out
# Installed by PCP QA test $seq on `date`
openbsd	139	dso	openbsd_init	$PCP_PMDAS_DIR/openbsd/pmda_openbsd.so
pmcd	2	dso	pmcd_init	$PCP_PMDAS_DIR/pmcd/pmda_pmcd.so
End-of-File
    else
	echo "Arrgh ... need pmcd.conf for $PCP_PLATFORM"
	exit 1
    fi

    $sudo cp $tmp.out $PCP_PMCDCONF_PATH

    echo | tee -a $seq_full
    echo "=== simple case, no frills, no errors ===" | tee -a $seq_full
    _poke_pmcd_n_logger

    echo | tee -a $seq_full
    echo "=== expected failure: a mere mortal ===" | tee -a $seq_full
    sudo_save="$sudo"
    unset sudo
    _poke_pmcd
    sudo="$sudo_save"
    echo "=== start again ===" | tee -a $seq_full
    _poke_pmcd_n_logger

    echo | tee -a $seq_full
    echo "=== expect 2 x connect failure: pmcd is true(1), so hard for pmlogger ===" | tee -a $seq_full
    $sudo rm -f $PCP_PMCD_PROG.$seq
    $sudo mv $PCP_PMCD_PROG $PCP_PMCD_PROG.$seq
    $sudo cp `which true` $PCP_PMCD_PROG
    _poke_pmcd_n_logger
    $sudo mv $PCP_PMCD_PROG.$seq $PCP_PMCD_PROG
    echo "=== start again ===" | tee -a $seq_full
    _poke_pmcd_n_logger

    echo "=== expect difficulty terminating pmcd ... it has been sent a SIGSTOP ===" | tee -a $seq_full
    pid=`_get_pids_by_name pmcd`
    if [ -z "$pid" ]
    then
	echo "Arrgh ... cannot find PID for pmcd!"
	ps $PCP_PS_ALL_FLAGS
	exit
    fi
    #DEBUG# ps $PCP_PS_ALL_FLAGS

    $sudo kill -STOP $pid
    _poke_pmcd

    echo | tee -a $seq_full
    echo "=== expect warning: PMLOGGER.CHECK moved, so no pmlogger ===" | tee -a $seq_full
    $sudo rm $CHECK
    _poke_pmcd_n_logger
    $sudo cp -p $CHECK.$seq $CHECK

    echo | tee -a $seq_full
    echo "=== expect error: pmlogger/control moved, so no pmlogger ===" | tee -a $seq_full
    $sudo rm -f $PCP_PMLOGGERCONTROL_PATH
    _poke_pmcd_n_logger
}

# real QA test starts here
_save_setup

if ! _service pmlogger stop 2>&1; then _exit 1; fi \
| _filter_pcp_stop
_wait_pmlogger_end || _exit 1
if ! _service pmcd stop 2>&1; then _exit 1; fi \
| _filter_pcp_stop
_wait_pmcd_end || _exit 1

( $sudo $PCP_RC_DIR/pmcd restart; $sudo $PCP_RC_DIR/pmlogger restart ) 2>&1 \
| tee -a $seq_full \
| _filter_pcp_start
_exercise_rc_scripts

# success, all done
status=0
exit
